package com.nasa.spider.webmagic.example;

import com.company.nuwa.common.utils.JsonUtils;
import com.google.common.collect.Lists;
import com.nasa.spider.webmagic.pipeline.FilePipeline;
import com.nasa.spider.webmagic.pipeline.JsonFilePipeline;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.pipeline.ConsolePipeline;
import us.codecraft.webmagic.pipeline.ResultItemsCollectorPipeline;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.List;

/**
 * <p></p >
 *
 * @author youyoukele
 * @version 1.0
 * @date 2023/7/5 16:43
 */
public class DianpingPageProcessor implements PageProcessor {

    private final Site site = Site.me()
            .setDomain(".dianping.com")
            .addHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
            .addHeader("Accept-Encoding", "gzip, deflate, br")
            .addHeader("Cookie", "fspop=test; cy=1; cye=shanghai; _lx_utm=utm_source%3Dgoogle%26utm_medium%3Dorganic; _lxsdk_cuid=189291b202ac8-025e986d922681-1b525634-1fa400-189291b202ac8; _lxsdk=189291b202ac8-025e986d922681-1b525634-1fa400-189291b202ac8; _hc.v=5bae44fa-d884-d534-be14-792560da1d1c.1688611791; Hm_lvt_602b80cf8079ae6591966cc70a3940e7=1688611792; s_ViewType=10; WEBDFPID=w4532638y66855vw1uz84222w8x24y79810708y126z97958z6wy2wz6-2003971814810-1688611813091CGCCEIY75613c134b6a252faa6802015be905513855; qruuid=27d99a24-67f4-40fc-bc18-48ae8f633273; dplet=f7b91f2643cbe32022d9bb7d5a09550e; dper=88590eab4440a7df284934a7928c6b41967bb612ec25d83f8edbdda9e9cc43c4699f8a67a1f3d7f0892e284e28dd645f6ec1a8f74e7e1318914df415fbc1f670; ll=7fd06e815b796be3df069dec7836c3df; ua=%E5%BF%A7%E5%BF%A7%E5%8F%AF%E4%B9%90_4619; ctu=1a632236a23150524c53bd30606dee3d40ec80ecc379d00894f24f8dfabb3560; _lxsdk_s=189291b202a-415-7a4-03a%7C%7C82; Hm_lpvt_602b80cf8079ae6591966cc70a3940e7=1688612101")
            .addHeader("Host", "www.dianping.com")
            .setUserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36")
            .setRetrySleepTime(100)
            .setRetryTimes(3);

    @Override
    public void process(Page page) {
        //page.putField("data", page.getHtml().xpath("//div[@class='txt']").all());
        page.putField("name", page.getHtml().xpath("//div[@class='txt']/div[@class='tit']/a/h4/text()").all());
        page.putField("price", page.getHtml().xpath("//div[@class='txt']/div[@class='comment']/a[@class='mean-price']/b/text()").all());
        page.putField("tag", page.getHtml().xpath("//div[@class='txt']/div[@class='tag-addr']/a[@data-click-name='shop_tag_cate_click']/span/text()").all());
        page.putField("region", page.getHtml().xpath("//div[@class='txt']/div[@class='tag-addr']/a[@data-click-name='shop_tag_region_click']/span/text()").all());

    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        List<String> urls = Lists.newArrayList(
                "https://www.dianping.com/shanghai/ch50/g183",
                "https://www.dianping.com/shanghai/ch50/g183p2",
                "https://www.dianping.com/shanghai/ch50/g183p3",
                "https://www.dianping.com/shanghai/ch50/g183p4",
                "https://www.dianping.com/shanghai/ch50/g183p5",
                "https://www.dianping.com/shanghai/ch50/g183p6",
                "https://www.dianping.com/shanghai/ch50/g183p7",
                "https://www.dianping.com/shanghai/ch50/g183p8",
                "https://www.dianping.com/shanghai/ch50/g183p9",
                "https://www.dianping.com/shanghai/ch50/g183p10"
        );

        ResultItemsCollectorPipeline resultItemsCollectorPipeline = new ResultItemsCollectorPipeline();
        Spider.create(new DianpingPageProcessor())
                .addUrl(urls.toArray(new String[0]))
                .addPipeline(new ConsolePipeline())
                .addPipeline(new FilePipeline("/Users/youyoukele/workspace/"))
                .addPipeline(new JsonFilePipeline("/Users/youyoukele/workspace/"))
                .addPipeline(resultItemsCollectorPipeline)
                .thread(10)
                .run();
        System.out.println(JsonUtils.objectToJson(resultItemsCollectorPipeline.getCollected()));
    }
}
